import re
import requests
from pyquery import PyQuery as pq
from sql_helper import MySqlHelper
import json
import time
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1.txt; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
headers = {
    'User-Agent': USER_AGENT
}


def read_jd_list(url, item_dict_all=None):
    if item_dict_all is None:
        item_dict_all = list()

    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        jsons = json.loads(response.text)

        for job in jsons['data']['results']:
            list_1 = list()
            url = job['positionURL']
            res = requests.get(url, headers=headers)
            html = pq(res.text)
            zhizhe = html(
                'body > div.wrap > div.main > div.pos-info.cl > div.l.pos-info-in > div.responsibility.pos-common > div.pos-ul').text()
            xueli = html(
                'body > div.wrap > div.main > div.main1.cl.main1-stat > div > ul > li.clearfix > div.info-three.l > span:nth-child(3)').text()
            dizhi = html(
                'body > div.wrap > div.main > div.pos-info.cl > div.l.pos-info-in > div.pos-common.work-add.cl > p.add-txt').text()
            list_1.append(job['jobName'])  # 岗位
            list_1.append(job['company']['name'])  # 公司名称
            list_1.append(dizhi)  # 地址
            list_1.append(job['city']['items'][0]['name'])  # 城市
            list_1.append(job['salary'])  # 薪水
            list_1.append(zhizhe)  # 岗位职责
            list_1.append(job['workingExp']['name'])  # 工作经验
            list_1.append(xueli)  # 学历
            list_1.append(str(job['welfare']).replace("[", '').replace("]", '').replace("\'", '').replace(",", ''))  # 福利
            item_dict_all.append(list_1)

    else:
        print("你抓的出问题了！")

    return item_dict_all


if __name__ == "__main__":
    # 北京 上海，广州 杭州 深圳 ,武汉,重庆 成都,郑州  西安, 济南, 长沙
    cs = ['530', '538', '763', '653', '765', '736', '551', '801', '854', '702', '749']
    for i in cs:
        url = "https://fe-api.zhaopin.com/c/i/sou?pageSize=90&cityId="+i+"&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=python&kt=3&_v=0.70152687&x-zp-page-request-id=dcf89c44ac8547828056fe2966ed646a-1546916540381-722141"
        item_dict_all = read_jd_list(url)
        mysql = MySqlHelper()
        # 岗位, 公司名称, 地址, 城市, 工资, 任职条件, 经验, 学历, 待遇
        sql = "INSERT INTO zlian(`post`, `company`, `site`, `city`, `salary`, `condition`, `experience`, `education`, `welfare`) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s)"
        
        mysql.exec_many(sql, item_dict_all)
